% scribe: Hua Chen
% lastupdate: Oct. 5, 2005
% lecture: 8
% title: Almost sure limits for sum of independent random variables
% references: Durrett, section 1.7
% keywords: almost sure convergence, SLLN, strong law of large numbers, Kolmogorov's maximal inequality, Kolmogorov's inequality
% end



\documentclass[12pt,letterpaper]{article}

\include{macros}

\begin{document}
\lecture{8}{Almost sure limits for sums of independent random
variables}{Hua Chen}{hchen@berkeley.edu}

This set of notes is a revision of the work of Animesh Kumar,
2002.

\section{Showing almost sure convergence via subsequences}
% keywords: almost sure convergence, SLLN, strong law of large numbers
% end

General settings/notation: let
$S_n=X_1+...+X_n$.  The $X_i$'s are assumed independent, all defined
on some probability space, $(\Omega,\mathcal{F},\P)$.  Sometimes, we
assume the $X_i$ are identically distributed.

An important technique for showing almost sure convergence from convergence
in probability is to consider subsequences.  We first note a few general facts 
about the various types of convergence we know:

\begin{enumerate}
\item If $Y_n \rightarrow Y \mbox{ a.s.}$ then $Y_n \rightarrow Y 
\mbox{ in $\P$}$.
\item If $Y_n \rightarrow Y \mbox{ in $\P$}$ then there exists a fixed
increasing subsequence $n_k$ such that $Y_{n_k} \rightarrow Y 
\mbox{ a.s.}$
\item $Y_n \rightarrow Y \mbox{ in $\P$}$  iff for every subsequence $n_k$
there exists a further subsequebce $n_{k}'$ so that $Y_{n_{k}'}
\rightarrow Y \mbox{ a.s.}$
\end{enumerate}

Proofs of 2 and 3 are in the textbook. We first begin with a
technique which uses the information about almost sure convergence
of a subsequence of a sequence of random variables, and then somehow
getting control over a maximum.  Let us now describe the technique.

One can prove $Y_n \rightarrow Y \mbox{ a.s.}$  by first
showing $Y_{n_k} \rightarrow Y \mbox{ a.s.}$ for some $n_k$ (we
choose $n_k$) and then getting control over
\begin{eqnarray*}
M_k = \max_{n_k \leq m < n_{k+1}} |X_m - X_{n_k}|
\end{eqnarray*}
In particular we must be able to show that $ M_k \rightarrow 0 \mbox{ a.s.}$
because if $\omega \in \Omega$ is such that both 
$X_{n_k}(\omega) \rightarrow 0$
and $M_k(\omega) \rightarrow 0$ then we get (using the triangle inequality and 
th fact that the max is
greater than the elements of set over which maximum is taken)
\begin{eqnarray*}
X_m(\omega) \rightarrow X(\omega),
\end{eqnarray*}
so if $M_k\ascv 0$, then the above holds almost surely. 
To illustrate how to use
the technique, we start with the example of SLLN with
a second moment condition.

\begin{theorem} If $X, X_1, X_2, ...$ are i.i.d.\ random variables with $E(X) = \mu$, $E(X^2) < \infty$, and $S_n := X_1 +X_2 + ... + X_n$, then
\begin{equation}
\label{eqn:sllnwithvariance} \frac{S_n}{n} \rightarrow E(X)
\mbox{ a.s.}
\end{equation}
\end{theorem}

\begin{proof}
First we find a subsequence converging almost surely to the mean. For that we
use two tools:
\begin{itemize}
\item convergence in probability; and
\item the Borel-Cantelli lemma.
\end{itemize}

Without loss of generality, we can assume that $E(X)=0$. From
Chebyshev's inequality we get
%
\begin{eqnarray*}
\P\left( \left|\frac{S_n}{n}\right| > \epsilon\right) <
\frac{E(X^2)}{n\epsilon^2}.
\end{eqnarray*}
%
This means that $\frac{S_n}{n} \rightarrow 0 \mbox{ in $\P$}$. Notice that
$\sum_k \frac{1}{k^2}$ converges to a finite value, therefore for the
subsequence $n_k = k^2$ we get, using the Borel-Cantelli lemma,
%
\begin{eqnarray*}
\P\left( \left|\frac{S_{n^2}}{n^2}\right| > \epsilon \mbox{ i.o.} \right) = 0,
\end{eqnarray*}
%
which means that $\frac{S_{n^2}}{n^2} \rightarrow 0 \mbox{ a.s.}$

Now let us try to control $M_k$ as defined above. For convenience we define
%
\begin{eqnarray*}
D_n := \max_{n^2 \leq k < (n+1)^2} |S_k - S_{n^2}|
\end{eqnarray*}
%
for $n^2 \leq k < (n+1)^2$.  We have $|S_k| \leq |S_{n^2}| + D_n$ and
$\frac{1}{k} \leq \frac{1}{n^2}$, so we have the following inequality:
%
\begin{eqnarray*}
\left| \frac{S_k}{k}\right| \leq \left| \frac{S_{n^2}}{n^2}\right| +
\frac{D_n}{n^2}.
\end{eqnarray*}
%
Finally, using the definition of $M_k$, we get the following:
%
\begin{eqnarray*}
M_k &\leq& \max_{n^2 \leq k < (n+1)^2} \left| \frac{S_k}{k}\right| + \left|
\frac{S_{n^2}}{n^2}\right|\\
  & \leq & 2\left|\frac{S_{n^2}}{n^2}\right| + \frac{D_n}{n^2}.\\
\end{eqnarray*}
%
So all we need to prove is that $\frac{D_n}{n^2} \rightarrow 0 \mbox{ a.s.}$ 
Let us define a new quantity $T_m = S_{n^2 + m} - S_{n^2}$.
Therefore,
%
\begin{eqnarray*}
D_{n}^2 & =& \max_{1\leq m \leq 2n} T_{m}^2 \\
& \leq & \sum_{m = 1}^{2n} T_{m}^2.
\end{eqnarray*}
%
Taking expectations on both sides, we get that
%
\begin{eqnarray*}
E(D_{n}^2)& \leq &\sum_{m = 1}^{2n} m\sigma^2  = n(2n+1) \sigma^2\\
&\leq & 4n^2 \sigma^2,
\end{eqnarray*}
%
where $E(X^2) = \sigma^2$. Hence we get that
%
\begin{eqnarray*}
\P\left( \left|\frac{D_n}{n^2}\right| > \epsilon \right) &\leq&
\frac{E\left(\left(\frac{D_n}{n^2}\right)^2 \right)} {\epsilon^2}\\
&\leq & \frac{4\sigma^2}{n^2 \epsilon^2}.
\end{eqnarray*}

Applying the Borel-Cantelli lemma with the fact 
\begin{eqnarray*}
\sum_n \P\left( \left|\frac{D_n}{n^2}\right| > \epsilon \right) < \infty
\end{eqnarray*}
we get that $\frac{D_n}{n^2} \rightarrow 0 \mbox{ a.s.}$, which completes
the proof.
\end{proof}

\section{Kolmogorov's Maximal Inequality}
% keywords: Kolmogorov's maximal inequality, Kolmogorov's inequality
% end

Now we proceed to Kolmogorov's inequality. We formally state it
as follows.

\begin{theorem}[Kolmogorov's Inequality] 
Let $X_1, X_2, ...$ be independent with
$E(X_i) = 0$ and $\sigma_{i}^2 =  E(X_{i}^2) < \infty$, and define $S_k = X_1 +
X_2 + ... +X_k$. Then 
\begin{equation}
\label{eqn:kolmogorovinequality}
\P\left( \max_{1\leq k \leq n} |S_k| \geq \epsilon \right) \leq
\frac{E(S_{n}^2)}{\epsilon^2}.
\end{equation}
\end{theorem}

\begin{proof}
Decompose the event according to when we escape from the $\pm \epsilon$
strip. Let
\begin{eqnarray*}
A_k = \{|S_m| < \epsilon \mbox{ for } 1 \leq m < k \mbox{; } |S_k| \geq \epsilon\}
\end{eqnarray*}
%
In words, $A_k$ is the event that the first escape out of the 
$\epsilon$ strip occurs at the $k$th step. Also notice that all these
events are disjoint, and that $\bigcup_{k = 1}^{n} A_k \left\{ \max_{1\leq k \leq n} |S_k| \geq \epsilon \right\}$. Then,
%
\begin{eqnarray*}
E(S_{n}^2) \geq E\left( S_{n}^2 1\left(\bigcup_{k=1}^n A_k\right) \right) \sum_{k=1}^{n} E\left(S_{n}^2 1_{A_k}\right).
\end{eqnarray*}
%
We can split $S_{n}^2 = S_{k}^2 +  (S_{n} - S_k)^2 + 2S_k(S_n - S_k)$, and 
write
%
\begin{eqnarray*}
E\left(S_{n}^2 1_{A_k}\right) &=& E\left( S_{k}^2 1_{A_k}\right) + E\left(
(S_n-S_k)^2 1_{A_k} \right) + E\left(2(S_n-S_k)S_k 1_{A_k}\right)\\
&\geq & \epsilon^2 \P(A_k),
\end{eqnarray*}
%
where the first term is larger than $\epsilon^2$, the second term is always 
positive, and the third term is the expectation of a product of two independent
random variables witn mean $0$.

Finally, we put this into the summation to get
\begin{eqnarray*}
E(S_{n}^2) \geq \sum_{k=1}^n \P(A_k) \epsilon^2 = \P\left( \max_{1\leq k \leq n}
|S_k| \geq \epsilon \right) \epsilon^2,
\end{eqnarray*}
which easily leads to the result.  This completes the proof.
\end{proof}

We observe that the inequality is valid for any sequence of r.v.'s
$(X_1,...X_n)$ such that 
$$E\left(2(S_n-S_k)S_k 1_{A_k}\right) = 0.$$
This will lead to the definition in future lectures of a \emph{martingale
difference sequence}.

\end{document}